﻿#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Supplementary File S2 — analysis code
================================================================================
Pyraloidea (Lepidoptera) of a Mediterranean elevational gradient in Mersin
Province, Türkiye: an annotated checklist with 80 new provincial records

Hüseyinoğlu Y, Kılıç E, Kılıç A, Atak S & Bahçeci E

This script reproduces every quantitative result reported in the paper from the
occurrence dataset published as Supplementary File S1. The estimators and the
models are implemented directly from the published formulae rather than taken
from a package, so that the calculations can be checked line by line and do not
depend on the version of any library beyond NumPy and SciPy.

  Input   File S1  (Pyraloidea_FileS1_occurrence_DarwinCore.csv, UTF-8)
  Output  the figures quoted in Sections 3.1, 3.4 and 3.5, printed to stdout

Tested with Python 3.12, NumPy 2.4.4 and SciPy 1.17.1.
Run:  python3 analysis.py Pyraloidea_FileS1_occurrence_DarwinCore.csv
================================================================================
"""
import csv
import itertools
import random
import re
import sys
from collections import Counter, defaultdict

import numpy as np
from scipy import optimize, stats

# ------------------------------------------------------------------ constants
BELTS = [(150, 400), (401, 800), (801, 1200), (1201, 1800)]
ANALYSIS_FROM = 2021   # nights from this year on are used in the analyses
MIN_SPECIES = 5        # minimum species per night for the turnover analysis
N_RANDOM = 1000        # randomisations for rarefaction
SEED = 7

# genera whose species cannot be separated on external characters
PROVISIONAL_GENERA = {
    "Cadra", "Delplanqueia", "Ephestia", "Epischnia", "Homoeosoma",
    "Pempelia", "Pempeliella", "Phycitodes", "Eudonia", "Scoparia",
}


# ------------------------------------------------------------------- reading
def binomen(name):
    """Genus and species epithet, with the authorship stripped off."""
    return " ".join(re.sub(r"\s*[\(\[].*", "", name).split()[:2])


def belt_of(elevation):
    for low, high in BELTS:
        if low <= elevation <= high:
            return (low, high)
    raise ValueError(f"elevation {elevation} falls outside the belts")


def read(path):
    """Return the records, and the species recorded on each sampling night.

    A sampling night is one locality worked on one date; only one locality was
    worked per night, so the pair (county, date) identifies it uniquely.
    """
    with open(path, encoding="utf-8-sig") as handle:
        records = list(csv.DictReader(handle))

    nights = defaultdict(set)
    attributes = {}
    for record in records:
        key = (record["county"], record["eventDate"])
        nights[key].add(binomen(record["scientificName"]))
        attributes[key] = {
            "elevation": int(record["minimumElevationInMeters"]),
            "month": int(record["month"]),
            "year": int(record["year"]),
            "county": record["county"],
        }
    return records, dict(nights), attributes


# ---------------------------------------------------------------- estimators
def chao2(nights):
    """Incidence-based richness estimator (Chao 1987).

        S + ((T-1)/T) * Q1^2 / (2*Q2)

    with the bias-corrected form used when no species is recorded in exactly
    two sampling units.
    """
    incidence = Counter()
    for species in nights.values():
        for name in species:
            incidence[name] += 1
    S = len(incidence)
    T = len(nights)
    Q1 = sum(1 for n in incidence.values() if n == 1)
    Q2 = sum(1 for n in incidence.values() if n == 2)
    if Q2:
        return S + ((T - 1) / T) * Q1 ** 2 / (2 * Q2), S, Q1, Q2
    return S + ((T - 1) / T) * Q1 * (Q1 - 1) / 2, S, Q1, Q2


def chao1(records):
    """Abundance-based richness estimator, from the individual counts."""
    abundance = Counter()
    for record in records:
        if record["individualCount"]:
            abundance[binomen(record["scientificName"])] += int(record["individualCount"])
    S = len(abundance)
    f1 = sum(1 for n in abundance.values() if n == 1)
    f2 = sum(1 for n in abundance.values() if n == 2)
    if f2:
        return S + f1 ** 2 / (2 * f2), S, f1, f2
    return S + f1 * (f1 - 1) / 2, S, f1, f2


def rarefaction(nights, keys, replicates=N_RANDOM, seed=SEED):
    """Mean and standard deviation of accumulated richness, over random orders
    of the sampling units."""
    rng = random.Random(seed)
    accumulated = np.zeros((replicates, len(keys)))
    for i in range(replicates):
        order = list(keys)
        rng.shuffle(order)
        seen = set()
        for j, key in enumerate(order):
            seen |= nights[key]
            accumulated[i, j] = len(seen)
    return accumulated.mean(axis=0), accumulated.std(axis=0)


# ------------------------------------------------------------------- models
def poisson_glm(X, y, iterations=100):
    """Poisson GLM by iteratively reweighted least squares.

    Returned together with the Pearson dispersion, which is what tells us
    whether the Poisson assumption is tenable. For these data it is not.
    """
    beta = np.zeros(X.shape[1])
    for _ in range(iterations):
        mu = np.exp(np.clip(X @ beta, -20, 20))
        z = X @ beta + (y - mu) / mu
        beta = np.linalg.solve(X.T @ (X * mu[:, None]), X.T @ (mu * z))
    mu = np.exp(X @ beta)
    covariance = np.linalg.inv(X.T @ (X * mu[:, None]))
    se = np.sqrt(np.diag(covariance))
    p = 2 * (1 - stats.norm.cdf(np.abs(beta / se)))
    dispersion = np.sum((y - mu) ** 2 / mu) / (len(y) - X.shape[1])
    return beta, se, p, dispersion


def negative_binomial_glm(X, y):
    """Negative binomial GLM by maximum likelihood.

    The counts are overdispersed relative to the Poisson, so this is the model
    reported in the paper. The dispersion parameter alpha is estimated jointly
    with the coefficients.
    """
    def negative_log_likelihood(parameters):
        beta, alpha = parameters[:-1], np.exp(parameters[-1])
        mu = np.exp(np.clip(X @ beta, -20, 20))
        r = 1 / alpha
        return -np.sum(stats.nbinom.logpmf(y, r, r / (r + mu)))

    start = np.r_[poisson_glm(X, y)[0], np.log(0.2)]
    fit = optimize.minimize(negative_log_likelihood, start, method="BFGS")

    # Standard errors from the observed information matrix, computed by central
    # differences. The Hessian approximation that the optimiser carries is not
    # reliable enough to quote, and varies with the path taken to the optimum.
    def hessian(f, x, step=1e-5):
        n = len(x)
        H = np.empty((n, n))
        for i in range(n):
            for j in range(n):
                ei, ej = np.zeros(n), np.zeros(n)
                ei[i] = ej[j] = step
                H[i, j] = (f(x + ei + ej) - f(x + ei - ej)
                           - f(x - ei + ej) + f(x - ei - ej)) / (4 * step ** 2)
        return (H + H.T) / 2

    covariance = np.linalg.inv(hessian(negative_log_likelihood, fit.x))
    beta = fit.x[:-1]
    se = np.sqrt(np.diag(covariance)[:-1])
    p = 2 * (1 - stats.norm.cdf(np.abs(beta / se)))
    return beta, se, p, np.exp(fit.x[-1])


# ------------------------------------------------------------------ turnover
def sorensen(a, b):
    """Sørensen dissimilarity between two sets of species."""
    return 1 - 2 * len(a & b) / (len(a) + len(b))


def baselga(a, b):
    """Partition of the Sørensen dissimilarity into its turnover and
    nestedness components (Baselga 2010).

    The turnover component is the Simpson dissimilarity; the nestedness
    component is what is left of the total.
    """
    shared = len(a & b)
    only_a, only_b = len(a - b), len(b - a)
    denominator = shared + min(only_a, only_b)
    turnover = min(only_a, only_b) / denominator if denominator else 0.0
    total = (only_a + only_b) / (2 * shared + only_a + only_b) if (2 * shared + only_a + only_b) else 0.0
    return turnover, total - turnover, total


def partial_spearman(x, y, z):
    """Spearman correlation of x and y with the effect of z removed."""
    rxy = stats.spearmanr(x, y).statistic
    rxz = stats.spearmanr(x, z).statistic
    ryz = stats.spearmanr(y, z).statistic
    return (rxy - rxz * ryz) / np.sqrt((1 - rxz ** 2) * (1 - ryz ** 2))


# ---------------------------------------------------------------------- main
def main(path):
    records, nights, attributes = read(path)
    species = {binomen(r["scientificName"]) for r in records}
    specimens = sum(int(r["individualCount"]) for r in records if r["individualCount"])

    print("=" * 78)
    print("SECTION 3.1  Composition of the fauna")
    print("=" * 78)
    print(f"records {len(records)} | species {len(species)} | specimens {specimens} | "
          f"nights {len(nights)} | localities {len({r['locationID'] for r in records})}")

    by_subfamily = defaultdict(set)
    by_genus = defaultdict(set)
    for record in records:
        key = (record["family"], record["subfamily"])
        by_subfamily[key].add(binomen(record["scientificName"]))
        by_genus[key].add(record["genus"])
    for key in sorted(by_subfamily, key=lambda k: (k[0], -len(by_subfamily[k]))):
        print(f"  {key[0]:<10}{key[1]:<15}{len(by_subfamily[key]):>4} species"
              f"{len(by_genus[key]):>4} genera")

    abundance = Counter()
    for record in records:
        if record["individualCount"]:
            abundance[binomen(record["scientificName"])] += int(record["individualCount"])
    commonest = abundance.most_common(10)
    print(f"\n  commonest species: {commonest[0][0]} {commonest[0][1]} specimens "
          f"({commonest[0][1] / specimens * 100:.1f}%)")
    print(f"  ten commonest together: {sum(n for _, n in commonest)} "
          f"({sum(n for _, n in commonest) / specimens * 100:.1f}%)")
    print(f"  species known from one specimen: {sum(1 for n in abundance.values() if n == 1)}")

    localities = defaultdict(set)
    for record in records:
        localities[binomen(record["scientificName"])].add(record["locationID"])
    print(f"  species known from one locality: {sum(1 for v in localities.values() if len(v) == 1)}")

    print()
    print("=" * 78)
    print("SECTION 3.4  Distribution along the elevational gradient")
    print("=" * 78)
    print(f"{'belt (m)':<14}{'localities':>11}{'nights':>8}{'species':>9}{'exclusive':>11}")
    belts_of_species = defaultdict(set)
    for record in records:
        belts_of_species[binomen(record["scientificName"])].add(
            belt_of(int(record["minimumElevationInMeters"])))
    for low, high in BELTS:
        subset = [r for r in records if low <= int(r["minimumElevationInMeters"]) <= high]
        in_belt = {binomen(r["scientificName"]) for r in subset}
        exclusive = sum(1 for _, v in belts_of_species.items() if v == {(low, high)})
        print(f"{f'{low}-{high}':<14}{len({r['locationID'] for r in subset}):>11}"
              f"{len({(r['county'], r['eventDate']) for r in subset}):>8}"
              f"{len(in_belt):>9}{exclusive:>11}")
    print(f"{'':<14}{'':>11}{'':>8}{'':>9}"
          f"{sum(1 for v in belts_of_species.values() if len(v) > 1):>11}  (in more than one belt)")

    analysed = [k for k in nights if attributes[k]["year"] >= ANALYSIS_FROM]
    y = np.array([len(nights[k]) for k in analysed], dtype=float)
    elevation = np.array([attributes[k]["elevation"] for k in analysed]) / 1000
    month = np.array([attributes[k]["month"] for k in analysed], dtype=float)
    district = np.array([1.0 if attributes[k]["county"] == "Erdemli" else 0.0 for k in analysed])

    print(f"\n  species per night against elevation and month, {len(analysed)} nights")
    X = np.column_stack([np.ones(len(analysed)), elevation, month])
    _, _, _, dispersion = poisson_glm(X, y)
    print(f"    Poisson dispersion {dispersion:.2f} — the Poisson model is not tenable")
    beta, se, p, alpha = negative_binomial_glm(X, y)
    for name, b, s, pv in zip(["intercept", "elevation", "month"], beta, se, p):
        print(f"    negative binomial  {name:<10}beta={b:+.3f}  SE={s:.3f}  p={pv:.4f}")
    print(f"    alpha {alpha:.3f}")
    X2 = np.column_stack([np.ones(len(analysed)), elevation, month, district])
    beta2, se2, p2, _ = negative_binomial_glm(X2, y)
    print(f"    with district      elevation beta={beta2[1]:+.3f}  p={p2[1]:.4f}  "
          f"| district p={p2[3]:.4f}")

    for_turnover = [k for k in analysed if len(nights[k]) >= MIN_SPECIES]
    pairs = list(itertools.combinations(for_turnover, 2))
    total, turnover, nestedness = [], [], []
    d_elevation, d_month, same_belt = [], [], []
    for a, b in pairs:
        t, n, s = baselga(nights[a], nights[b])
        turnover.append(t)
        nestedness.append(n)
        total.append(s)
        d_elevation.append(abs(attributes[a]["elevation"] - attributes[b]["elevation"]))
        d_month.append(abs(attributes[a]["month"] - attributes[b]["month"]))
        same_belt.append(belt_of(attributes[a]["elevation"]) == belt_of(attributes[b]["elevation"]))
    total = np.array(total)
    turnover = np.array(turnover)
    nestedness = np.array(nestedness)
    d_elevation = np.array(d_elevation)
    d_month = np.array(d_month)
    same_belt = np.array(same_belt)

    print(f"\n  compositional turnover, {len(for_turnover)} nights, {len(pairs)} pairs")
    r_elev = stats.spearmanr(d_elevation, total)
    r_month = stats.spearmanr(d_month, total)
    print(f"    Sorensen against elevational distance  rho={r_elev.statistic:+.3f}  p={r_elev.pvalue:.2e}")
    print(f"    Sorensen against difference in month   rho={r_month.statistic:+.3f}  p={r_month.pvalue:.4f}")
    print(f"    partial, month held constant           rho={partial_spearman(d_elevation, total, d_month):+.3f}")
    u = stats.mannwhitneyu(total[same_belt], total[~same_belt])
    print(f"    within belts {total[same_belt].mean():.3f} against between belts "
          f"{total[~same_belt].mean():.3f}, Mann-Whitney p={u.pvalue:.3f}")
    print(f"    partition: total {total.mean():.3f} = turnover {turnover.mean():.3f} "
          f"({turnover.mean() / total.mean() * 100:.0f}%) + nestedness {nestedness.mean():.3f} "
          f"({nestedness.mean() / total.mean() * 100:.0f}%)")
    rt = stats.spearmanr(d_elevation, turnover)
    rn = stats.spearmanr(d_elevation, nestedness)
    print(f"    turnover component against elevation   rho={rt.statistic:+.3f}  p={rt.pvalue:.2e}")
    print(f"    nestedness component against elevation rho={rn.statistic:+.3f}  p={rn.pvalue:.4f}")

    print()
    print("=" * 78)
    print("SECTION 3.5  Completeness of the inventory")
    print("=" * 78)
    early = [k for k in nights if attributes[k]["year"] < 2015]
    print(f"{'set':<22}{'nights':>7}{'species':>9}{'Q1':>5}{'Q2':>5}{'Chao2':>8}{'recorded':>10}")
    for label, keys in [("all nights", list(nights)),
                        (f"{ANALYSIS_FROM}-2024", analysed),
                        ("2008-2014", early)]:
        subset = {k: nights[k] for k in keys}
        estimate, S, Q1, Q2 = chao2(subset)
        print(f"{label:<22}{len(keys):>7}{S:>9}{Q1:>5}{Q2:>5}{estimate:>8.0f}{S / estimate * 100:>9.0f}%")
    estimate, S, f1, f2 = chao1(records)
    print(f"\n  Chao1, all records: {estimate:.0f} against {S} observed (f1={f1}, f2={f2})")

    print("\n  rarefaction, mean accumulated species at selected numbers of nights")
    for label, keys in [("all nights", list(nights)),
                        (f"{ANALYSIS_FROM}-2024", analysed),
                        ("2008-2014", early)]:
        mean, sd = rarefaction(nights, keys)
        marks = [n for n in (5, 10, 20, 24, 30, 54) if n <= len(keys)]
        cells = "  ".join(f"{n}:{mean[n - 1]:.1f}" for n in marks)
        print(f"    {label:<20}{cells}")

    print("\n  provisional identifications")
    provisional = [r for r in records
                   if binomen(r["scientificName"]).split()[0] in PROVISIONAL_GENERA]
    ps = {binomen(r["scientificName"]) for r in provisional}
    pi = sum(int(r["individualCount"]) for r in provisional if r["individualCount"])
    print(f"    {len(ps)} species ({len(ps) / len(species) * 100:.1f}%), "
          f"{len(provisional)} records ({len(provisional) / len(records) * 100:.1f}%), "
          f"{pi} specimens ({pi / specimens * 100:.1f}%)")


if __name__ == "__main__":
    main(sys.argv[1] if len(sys.argv) > 1
         else "Pyraloidea_FileS1_occurrence_DarwinCore.csv")

